#encoding=utf-8
import urllib2
import re

class Spider:
    '''
    内涵段子爬虫
    '''
    def __init__(self,page):
        self.enable = 1
        self.page = page
    def loadPage(self,page):
        """
        :description:定义一个url请求页面的方法
        :param page: 需要请求的是第几也
        :return: 返回页面html
        """

        url = "https://www.neihan8.com/article/list_5_" + str(page) +'.html'
        #User-Agent头
        user_agent = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0"

        headers = {'User-Agent':user_agent}
        req = urllib2.Request(url, headers=headers)
        response = urllib2.urlopen(req)
        html = response.read()
        gbk_html = html.decode('gbk').encode('utf-8')

        #找到所有的段子内容<div class = "f18 mb20"></div>
        #re.S如果没有re.S则只是匹配一行有没有符合规则的字符串，如果没有则下一行重新匹配
        #如果加上re.S则是将所有的字符串做一个整体进行匹配
        pattern = re.compile(r'<div.*?class="f18 mb20">(.*?)</div>',re.S)
        item_list = pattern.findall(gbk_html)
        #print gbk_html
        #print item_list
        return item_list

        #return  html

    def printOnePage(self,item_list,page):
        print "******* 第 %d 页 爬取完毕...*******" %page
        for item in item_list:
            #print "============"
            item = item.replace("<p>","").replace("</p>","").replace("&ldquo","").replace("&rdquo","").replace("<br />","").replace("&hellip","").replace(";",'')
            self.writeToFile(item)

    def writeToFile(self,text):
        """
        :description:将数据追加写进文件中
        :param text: 文件内容
        :return:
        """
        myFile = open("./duanzi.txt",'a')
        myFile.write(text)
        myFile.write("----------------------------------------------------")
        myFile.close()

    def doWork(self):
        while self.enable:
            try:
                item_list = self.loadPage(self.page)
            except urllib2.URLError,e:
                print e.reason
                continue

            # 对得到的段子item_list处理
            self.printOnePage(item_list,self.page)
            self.page += 1
            print "按回车继续。。。。"
            print "输入quit退出"
            command = raw_input()
            if (command == "quit"):
                self.enable = False
                break


def main():
    """
    内涵段子小爬虫
    :return:
    """
    print "请按下回车开始"
    raw_input()
    #定义一个Sprider对象
    mysprider = Spider(1)
    mysprider.doWork()
if __name__ == '__main__':
    main()